import os
from scipy import sparse

#-----------------------------------------------------------------------------#
#                         Parametres externes :                               #
#-----------------------------------------------------------------------------#
 
NOISE = ['\n', ',', '.', ';', ':', '(', ')', '[', ']', '"', '?', '&', '$', '?'\
         , '}', '{', '!', '+', '~', '=', '!', '*', '\\', '/', '--']

MIN_FREQ_WORD = 1
MIN_FREQ_CAT = 0

#-----------------------------------------------------------------------------#
           
def tokenize(slice):
    """
    Cette fonction sert a enlever du bruit.
    """
    slice = slice.lower()
    for i in NOISE:
        slice = slice.replace(i, ' ')
    return set(slice.split(' '))


def readStopWords():
    """
    Lecture du fichier des stopwords.
    """
    f = open('../stopwords.csv', 'r')
    b = f.read()
    f.close()
    b = b.replace('"', '')
    b = b.split('\n')
    return set(b)


class Categories(object):
    """
    Classe contennant un dictionnaire des objets categories, qui sert a leur associer :
    leur id et leur frequence.
    
        {c1:[id,freq], ..., cn:[id,freq]}
    """
    def __init__(self):
        self.nb = 0
        self.bag = {}

    def add(self, c):
        if self.bag.has_key(c):
            self.bag[c][1] = self.bag[c][1] + 1
        else:
            self.bag[c] = [self.nb, 1]
            self.nb = self.nb + 1        
        
        
class Words(object):
    """
    Classe contennant un dictionnaire des mots retrouves, qui sert a associer aux
    mots leurs ids et leur frequence:
    
        {w1:[id,id2,freq], ..., wn:[id,id2,freq]}
    """
    LOW_FREQ = -1
    ID = 0
    ID2 = 1
    FREQUENCE = 2    
    
    def __init__(self):
        self.nb = 0
        self.nbHighFrequence = 0     
        self.bag = {}
        
    def add(self, w):
        if self.bag.has_key(w):
            self.bag[w][Words.FREQUENCE] = self.bag[w][Words.FREQUENCE] + 1            
            if self.bag[w][Words.ID2] == Words.LOW_FREQ and self.bag[w][Words.FREQUENCE] > MIN_FREQ_WORD:
                self.bag[w][Words.ID2] = self.nbHighFrequence
                self.nbHighFrequence = self.nbHighFrequence + 1
        else:
            self.bag[w] = [self.nb, Words.LOW_FREQ, 1]
            self.nb = self.nb + 1

    def addWords(self, words):
        for i in words:
            self.add(i)
    
        
            
class Files(object):
    """
    Classe contennant un dictionnaire des fichiers retrouves, qui sert a associer aux
    fichiers leur id:
    
        {f0:[id], ..., fn:[id]}
    """
    def __init__(self):
        self.idFile = 0;
        self.bag = {}

    def add(self, file):
        self.bag[file] = self.idFile
        self.idFile = self.idFile + 1            

        
def readCategory(a):
    return a[a.find('NSF Program : ') + 24:a.find('NSF Program : ') + a[a.find('NSF Program : '):].find('\n')]


def readWords(stopwords, a):
    return tokenize(a[a.find('\nAbstract    :\n') + '\nAbstract    :\n'.__len__():]) - stopwords


def readFileData(path, filename):
    f = open(path + '/' + filename, 'r')
    fileData = f.read()
    f.close()
    return fileData


def readFiles(path, allwords, categories, stopwords, files):
    """
    Cette fonction sert fileData lire les fichiers et extraire les categories et les retrouves,
    afin de generer les dictionnaires Words et Categories.
    """
    for i in os.listdir(path):
        if i == 'awards_1990' or i == 'awards_1991' or i == 'awards_1992' :
            newPath1 = path + '/' + i
            for j in os.listdir(newPath1):
                if j != 'links.html':
                    newPath2 = newPath1 + '/' + j
                    for k in os.listdir(newPath2):
                        if k != 'links.html':
                            files.add(k)
                            fileData = readFileData(newPath2, k)
                            allwords.addWords(readWords(stopwords, fileData))
                            categories.add(readCategory(fileData))


def wordsToMatrix(allwords, words, nbFile, matrix):
    """
    Cette fonction sert a remplir une file de la matrice.
    """
    for i in words:
        if allwords.bag[i][Words.FREQUENCE] >= MIN_FREQ_WORD:
            col = allwords.bag[i][1] + 2
            matrix[nbFile, col] = 1


def loadMatrix(path, allwords, categories, stopwords, files):
    """
    Cette fonction sert fileData construire la matrice sparse qui contiendra tous les donnees
    
    |idFichier|idCategorie|frequence du mot1|...|frequence du motn| 
    
    ou n est le nombre des mots retrouves parmi l'ensemble des fichier lus.
    
    """
    matrix = sparse.lil_matrix((files.bag.__len__(), allwords.nbHighFrequence + 2), dtype='int32')
    for i in os.listdir(path):
        if i == 'awards_1990' or i == 'awards_1991' or i == 'awards_1992' :
            newPath1 = path + '/' + i
            for j in os.listdir(newPath1):
                if j != 'links.html':
                    newPath2 = newPath1 + '/' + j
                    for k in os.listdir(newPath2):
                        if k != 'links.html':
                            fileData = readFileData(newPath2, k)
                            category = readCategory(fileData)
                            words = readWords(stopwords, fileData)
                            wordsToMatrix(allwords, words, files.bag[k], matrix)
                            matrix[files.bag[k], 0] = files.bag[k]
                            matrix[files.bag[k], 1] = categories.bag[category][0]                       
    return matrix


def saveMatrix(matrix, filename):
    matrix.tocoo().save(filename ,format='%d %d %f\n')


def openMatrixFile(path, filename):
    fileData = readFileData(path, filename);
    fileData = fileData.split('\n')
    fileData[0] = map(lambda x: int(x),fileData[0].split(' '))
    newMatrix = sparse.lil_matrix((fileData[0][0],fileData[0][1]), dtype='int32')
    fileData[1] = int(fileData[1])
    for i in range(fileData[1]):
        c = fileData[i+2].split(' ')
        newMatrix[int(c[0]),int(c[1])] = float(c[2])
    return newMatrix

#-----------------------------------------------------------------------------#


"Instances de Words"
allwords = Words()

"Instance de Categories"
categories = Categories()

"Ensemble des stopwords"
stopwords = readStopWords()

"Instance de Files"
files = Files()

"Nombre des fichiers lus"
readFiles('../../Part1', allwords, categories, stopwords, files)

"Chargement de la matrice"
matrix = loadMatrix('../../Part1', allwords, categories, stopwords, files)

"Sauvegarde de la matrice dans un fichier de texte"
#saveMatrix(matrix, 'm.txt')

"On efface la matrice"
#del matrix


